# -*- coding: UTF-8 -*-
import pandas as pd
import time,os,datetime,math,struct,sys
from colorama import init, Fore, Back, Style
# from makedir import FileOp
pd.set_option('display.expand_frame_repr', False)
init(autoreset=True)
import logging
from loguru import logger
from struct import *
# from Tools import set_title
# from read_ini_config import Config
# logger.add('my_log.log')
currentDate = time.strftime("%Y%m%d", time.localtime())
LOG_DIR = os.path.expanduser("logs/")
LOG_FILE = os.path.join(LOG_DIR, currentDate+".log")
# if os.path.exits(LOG_DIR):
#     os.mkdir(LOG_DIR)
# 只输出到文本，不在console输出
# logger.remove(handler_id=None)
# logger.add("file.log", format="{time} {level} {message}", filter="", level="INFO")
logger.add(LOG_FILE,rotation = "2000KB" , encoding='gb2312')   #压缩成zip格式
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
#                                   class STOCK:
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
class STOCK:
    # ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////getlist start/////////
    def __init__(self):
        print("STOCK")
    @staticmethod
    def getlist(pathdir):
        listfile = os.listdir(pathdir)
        # example
        suc="初始化读取"+ pathdir+ "目录"+ str(len(listfile))+"个文件"
        logger.info(suc)
        return listfile,'success'

    @staticmethod
    def get_data(exeTag,dirpath,target_dir,color_para,funcxxx, **kwargs):

        stocklist, msg = STOCK.getlist(dirpath)
        scale =len(stocklist)
        # print()
        print(Fore.WHITE+Style.BRIGHT+Back.BLACK+ "获取文件数量["+str(scale)+"]\n读取目录:"+dirpath+"\n导出文件目录"+target_dir+"\n")
        i=0
        LL=40
        count_file=0
        # fo = open(LOG_FILE, "a")

        for list_filename in stocklist:
            i += 1
            msgfun_name=getattr(funcxxx,'__name__')
            df,msg1,msg2,msg3=funcxxx(exeTag,dirpath,list_filename,target_dir,color_para,**kwargs)

            count_file += 1
            if(type(df)==int):
                count_file=count_file-1

            a = (i / scale) * 100
            b = '.' * LL
            c = '▋' * (int((i * LL) / scale))
            b = '.' * ((LL-1) - int((i * LL) / scale))
            dur = time.perf_counter()  # //每次获取当前时间
            outmessage=","+'MESSAGE:' + msg1+",IN:"+msg2 +"--> OUT:"+msg3+",FILE COUNT:"+str(count_file)
            # fo.writelines(outmessage+ '\n')
            print(Style.BRIGHT+color_para+"\r[{}]\t{:^3.0f}% {}{} {:.2f}s".format(i, a, c, b, dur),Back.GREEN+"funname:"+msgfun_name,Fore.GREEN+outmessage,end=' ')

            # time.sleep(0.0001)
            # if i==1:
            #     break
        # fo.close()
        print('\n')
        return count_file
#=======================================================================================================================funcx1() START....................
def funcx1(tag,filepath,filename,target_dir,color_para):
    info =color_para+"目录:"+filepath+"文件名:"+filename
    # print(info)
    filepath_final=filepath+filename

    msg1=""
    msg2=str(filename).upper()
    target_file=str(filename).upper()+".CSV"
    expoer_file=target_dir+target_file
    # print(filepath_final,expoer_file)
    if tag == "DAY":
        df = TdxTxtDF.read_data(filepath_final, "DAY")
        if type(df) != int:
            # print(df)
            df.to_csv(target_dir + filename[0:10] + "DAY.CSV", index=True)
        #
    if tag == "5MIN":
        df = TdxTxtDF.read_data(filepath_final, "5MIN")
        if type(df) != int:
            df.to_csv(target_dir + filename[0:10] + "5MIN.CSV", index=True)

    if tag == "B":
        df = TdxDataDF.read_data(filepath+filename)
        if type(df) != int:
            df.to_csv(expoer_file, index=True)

    # print(df.tail(2))
    msg3=target_file

    return df,msg1,msg2,msg3


#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def set_title():
    fo = open("banner.txt", "r")
    # print("文件名为: ", fo.name)
    strtmp = ""
    for line in fo.readlines():  # 依次读取每行
        # line = line.strip()  # 去掉每行头尾空白
        # print("读取的数据为: %s" % (line))
        strtmp += line

    # 关闭文件

    fo.close()
    fo = open(LOG_FILE, "a")
    fo.writelines(strtmp)
    fo.close()
    return strtmp

#////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////printinfo
def printinfo(text):
    print(text)
    fo = open(LOG_FILE, "a")
    fo.writelines(str(text)+'\n')
    fo.close()
#////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////printinfo
@logger.catch
def show_title(title,funcxxx, **kwargs):
    logger.info('show_title')
    # os.system("cls")
    starttime = datetime.datetime.now()
    logger.debug("开始调用funcxx函数。。。")
    print(set_title())
    funcxxx(title,**kwargs)

    endtime = datetime.datetime.now()
    logger.success('程序运行时间:'+ str(endtime - starttime))

    # print("="*150)
@logger.catch
def set_running(title):
    logger.info('set_running')
    cf = Config("config.ini")

    path_list=[]
    path_list.append(cf.get_config("IOFileConfig", "path_txt_csv1"))
    path_list.append(cf.get_config("IOFileConfig", "path_txt_csv2"))
    path_list.append(cf.get_config("IOFileConfig", "pathdir1"))
    path_list.append(cf.get_config("IOFileConfig", "pathdir2"))
    path_list.append(cf.get_config("IOFileConfig", "pathdir3"))
    path_list.append(cf.get_config("IOFileConfig", "pathdir4"))
    filesum=0
    filesize=0
    for li in path_list:
        filecount=len(os.listdir(li))
        result = FileOp.get_totalsize(li)
        filesize+=result
        resultstr="Total size:({}MB)".format(result // 1024 // 1024)
        # filesum+=result

        logger.success(li+"\t 共计:("+str(filecount)+")个文件"+resultstr)


    currentDate = time.strftime("%Y%m%d", time.localtime())
    currentDate = currentDate + "_final"
    targetDir = cf.get_config("IOFileConfig", "exportdir") + currentDate + "\\"

    logger.warning("所有目录总共：{:,}".format(filesum)+"个文件,")
    logger.warning("Total size:({}MB)".format(filesize // 1024 // 1024))
    logger.success(targetDir)

    FileOp.mkdir(targetDir)
    FileOp.deldir(targetDir)
    # logger.warning(Fore.RED + Style.BRIGHT + targetDir + "目录已经准备就绪，按任意键继续。。。")
    # strinput=input(Fore.RED + Style.BRIGHT + targetDir +"程序已经准备就绪。。。按1继续，按任意键退出！")
    # logger.success(type(strinput))
    # if strinput!="1":
    #     sys.exit(2)
    logger.warning("~"*100)
    x1 = 0
    # logger.warning("\n日线数据\n")

    x1 += STOCK.get_data("DAY", path_list[0], targetDir, Fore.RED, funcxxx=funcx1)

    # logger.warning("\n五分钟数据")
    x1 += STOCK.get_data("5MIN", path_list[1], targetDir, Fore.BLUE, funcxxx=funcx1)

    # logger.warning("\n上海日线")
    x1 += STOCK.get_data("B", path_list[2], targetDir, Fore.YELLOW, funcxxx=funcx1)
    # logger.warning("\n上海五分钟数据")
    x1 += STOCK.get_data("B", path_list[3], targetDir, Fore.CYAN, funcxxx=funcx1)
    # logger.warning("\n深圳日线")
    x1 += STOCK.get_data("B", path_list[4], targetDir, Fore.MAGENTA, funcxxx=funcx1)

    # logger.warning("\n深圳五分钟数据")
    x1 += STOCK.get_data("B", path_list[5], targetDir, Fore.LIGHTRED_EX, funcxxx=funcx1)


    listfile = os.listdir(targetDir)
    print("\n")
    if len(listfile)==x1:
        msg="successfully!"
    else:
        msg="error!"
    warnings="导出目录："+targetDir+str("\t文件总数:").upper()+str(x1)+"\t"+msg.upper()
    logger.success(warnings)
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@TdxTxtDF
class TdxTxtDF:
    # //////////////////////////////////////////get_df///////////////////////////////////////////////////////////////////get_df
    @staticmethod
    @logger.catch
    def read_data( pathFile, flag_day):
        # logger.success("读取TXT文件。。。")
        # print("\t get_df(",pathFile,self.bianma,flag_day,")")
        # ===========================判断文件头部===========================
        if flag_day == "DAY":
            titleName = ['trade_date', 'open', 'high', 'low', 'close', 'volume', 'amount']
        else:
            titleName = ['trade_date', 'trade_time', 'open', 'high', 'low', 'close', 'volume', 'amount']
        df = pd.read_table(pathFile, header=1, encoding="gbk", names=titleName,
                           converters={'trade_time': str, 'trade_date': str})
        # df = pd.read_table(pathFile, header=1, encoding="gbk")
        # ===========================判断文件头部===========================
        # df["trade_date"] = pd.to_datetime(df["trade_date"], format='%Y/%m/%d')
        # df["trade_date"] = pd.to_datetime(df["trade_date"], errors='coerce', format='%Y%m%d')
        # print(df["trade_date"])
        list_firstline = TdxTxtDF.read_file_one(pathFile).split()
        # 删除最后一行SZ#399007.txt 5254    数据来源:通达信   NaN   NaN   NaN    NaN          NaN           NaN  1  1
        df.drop([len(df) - 1], inplace=True)
        if df.shape[0] < 1:
            # os.system("pause")
            return 0
        if flag_day != "DAY":
            # 清理1300的数据
            df.loc[df['trade_time'] == '1300', 'trade_time'] = 1130



        # df["trade_date"] = pd.to_datetime(df["trade_date"], format='%Y-%m-%d')
        # 分割年月日
        spl_year = df["trade_date"].str.split("/", expand=True).iloc[:, 0]
        spl_month = df["trade_date"].str.split("/", expand=True).iloc[:, 1]
        spl_day = df["trade_date"].str.split("/", expand=True).iloc[:, 2]
        df["trade_date"] = spl_year + spl_month + spl_day

        if flag_day != "DAY":
            #     spl_hou = df["Trade_times"].str.split("", expand=True).iloc[:, 0]
            #     spl_min = df["Trade_times"].str.split("", expand=True).iloc[:, 1]
            #     df["trade_times"] =df["trade_times"]
            df["trade_time"] = df["trade_time"].astype('str').str[0:2] + ":" + df["trade_time"].astype('str').str[
                                                                                 2:4] + ":00"
            # df["trade_times"] = pd.to_datetime(df["trade_times"], format='%H:%M:S')

        # 添加TS_CODE列/////////////////////////////////////////////
        tscode = TdxTxtDF.get_tscode(pathFile)
        df.insert(0, 'ts_code', tscode, allow_duplicates=False)

        pdweek = pd.to_datetime(df.trade_date, format='%Y-%m-%d %H:%M:%S.%f')
        # print(pdweek.dt.dayofweek+1)
        if flag_day == "DAY":
            df.insert(2, 'trade_time', pdweek.dt.dayofweek + 1, allow_duplicates=False)

        df = TdxTxtDF.set_col(df, list_firstline)
        df.columns = ['ID'] + df.columns[1:].tolist()
        df.columns = df.columns.str.upper()
        # print(df.tail(50))
        return df

    # //////////////////////////////////////////get_df///////////////////////////////////////////////////////////////////get_tscode
    # 转换路径为 XXXXXX.SZ 代码 E:\stcokData\20220116\stock_5MIN_qfq\SH#600000.txt
    @staticmethod
    def get_tscode( pathFile):
        pathFile = pathFile[-13:]
        mk = pathFile[:2]
        code = pathFile[3:][0:6]
        tscode = code + "." + mk
        mk = mk
        # 返回格式 XXXXXX.SZ
        return tscode


    # ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////read_fileone
    # 读取文件的一行数据
    @staticmethod
    def read_file_one( filename):
        # 打开文件
        fo = open(filename, "r")
        fileFirstLine = fo.readline()
        fo.close()
        return fileFirstLine

    @staticmethod
    def set_col(df, list_firstline):

        df.insert(df.shape[1], 'period', list_firstline[2], allow_duplicates=False)
        df.insert(df.shape[1], 'adj', list_firstline[3], allow_duplicates=False)
        # currentDate = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        df.insert(df.shape[1], 'tag', "原始数据", allow_duplicates=False)
        df.insert(df.shape[1], 'source', "TDX_TXT", allow_duplicates=False)
        return df

#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ TdxDataDF

class TdxDataDF:

    @staticmethod
    @logger.catch
    def read_data(filename):
        # logger.success("读取"+filename+"二进制文件")
        tmpcode=filename[-12:]
        tscodetmp =tmpcode[0:8]
        mk=tmpcode[0:2]
        codetmp=tmpcode[2:8]
        tscode=codetmp+"."+mk.upper()
        # print(mk,codetmp)
        ofile = open(filename, 'rb')
        buf = ofile.read()
        ofile.close()
        num = len(buf)
        no = num // 32
        b = 0
        e = 32
        dl = []
        tmpTag=filename[-3:]
        # print(tmpTag)
        for i in range(no):
            if tmpTag=="lc5":
                a = unpack('HHfffffii', buf[b:e])   #hhfffffii
                xxx = a[0]
                year_str = str(math.floor(int(xxx) / 2048) + 2004)
                month_str=str(int(xxx % 2048 / 100)).zfill(2)
                day_str=str(xxx % 2048 % 100).zfill(2)
                date_str = str(year_str+month_str+day_str)
                time_str =str(int(a[1] / 60)).zfill(2) + ':' + str(a[1] % 60).zfill(2) + ':00'
                dl.append([tscode,date_str,time_str,str(TdxDataDF.c_round(a[2])),str(TdxDataDF.c_round(a[3])),str(TdxDataDF.c_round(a[4])),str(TdxDataDF.c_round(a[5])),str(a[6]),str(a[7]),str(a[8]),tmpTag])

            if tmpTag == "day":
                a = unpack('IIIIIfII', buf[b:e])
                xxx = a[0]
                year_str=str(xxx)[0:4]
                month_str=str(xxx)[4:6]
                day_str=str(xxx)[-2:]
                date_str = str(year_str+month_str+day_str)
                # week = datetime.datetime.strptime(date_str, "%Y-%m-%d").weekday() + 1

                date_format = datetime.datetime.strptime(TdxDataDF.c_date(date_str), '%Y-%m-%d').weekday() + 1  # 格式化日期

                time_str=str(date_format)
                close=a[4]/ 100.0
                if i == 0:
                    preClose = close
                ratio = round((close - preClose) / preClose * 100, 2)

                dl.append([tscode,date_str,time_str,str(a[1]/ 100.0),str(a[2]/ 100.0),str(a[3]/ 100.0),str(a[4]/ 100.0),str(a[5]),str(a[6]),str(a[7]) ,tmpTag])
                preClose = close
            b = b + 32
            e = e + 32
        titlename=['ts_code','trade_date', 'trade_time', 'open', 'high', 'low', 'close', 'amount', 'volume','tmp','period']
        df = pd.DataFrame(dl,columns=titlename)
        df.insert(df.shape[1], 'adj', "未复权", allow_duplicates=False)
        df.insert(df.shape[1], 'tag', "原始数据", allow_duplicates=False)
        df.insert(df.shape[1], 'source', "Data_TXT", allow_duplicates=False)
        df.columns = df.columns.str.upper()
        # df.drop(df.tmp)
        #删除多余的标注列
        df.drop(df.columns[9],axis=1, inplace=True)
        #在第一列加ID字段
        df.columns = ['ID'] + df.columns[1:].tolist()
        return df
    @staticmethod
    def c_round(nums):
        return round(nums,3)

    @staticmethod
    #日期格式转化
    def c_date(date_str,geshi="-"):
        year_str = str(date_str)[0:4]
        month_str = str(date_str)[4:6]
        day_str = str(date_str)[-2:]
        xxx = str(year_str +geshi+ month_str+geshi + day_str)
        # print("x",time.strptime(date_str, "%Y-%M-%d"))

        return xxx
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@FileOp@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@Config
import configparser
class Config:
    def __init__(self,filename,encoding="gbk"):
        print("def __init__(self):初始化",filename,encoding)
        self.cf = configparser.ConfigParser()
        self.cf.read(filename, encoding=encoding)  # 读取配置文件，如果写文件的绝对路径，就可以不用os模块
        # 获取文件中所有的section(一个配置文件中可以有多个配置，如数据库相关的配置，邮箱相关的配置，
        # 每个section由[]包裹，即[section])，并以列表的形式返回
    def get_config(self,rootstr,txt):
        return self.cf.get(rootstr, txt)
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@FileOp@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@FileOp
class FileOp:
    def mkdir(path):
        # 引入模块
        import os
        # 去除首位空格
        path = path.strip()
        # 去除尾部 \ 符号
        path = path.rstrip("\\")
        # 判断路径是否存在# 存在     True# 不存在   False
        isExists = os.path.exists(path)
        # 判断结果
        if not isExists:
            # 如果不存在则创建目录
            # 创建目录操作函数
            os.makedirs(path)
            logger.success(path + ' 创建成功')
            return True
        else:
            # 如果目录存在则不创建，并提示目录已存在
            logger.warning(path + ' 目录已存在')
            return False

    @staticmethod
    def get_totalsize(path_add):

        size = 0
        # 遍历并装载第一层的文件和文件夹
        list = os.listdir(path_add)

        for i in list:
            # 拼接成完整的绝对路径
            pathfile = os.path.join(path_add, i)
            # 文件 OR NOT
            if os.path.isfile(pathfile):
                # print(i," is FILE")
                # print(pathfile)
                # 统计文件大小
                size += os.path.getsize(pathfile)
            # 文件夹 OR NOT
            elif os.path.isdir(pathfile):
                # print(i," is FOLDER")
                # print(pathfile)
                # 递归统计文件夹里面的文件名称
                size += FileOp.get_totalsize(pathfile)

        return size


    @staticmethod
    def deldir(location):
        import glob
        # Importing os and glob modules
        # Loop Through the folder projects all files and deleting them one by one
        for file in glob.glob(location + "/*"):
            os.remove(file)
            logger.success("Deleted " + str(file))
    # result = getTotalsize(path_add)
    # print("Total size:{}字节({}MB)".format(result, result // 1024 // 1024))  # 输出总大小
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@__name__
def main():
    os.system("cls")
    logger.critical('=' * 100)
    logger.success("开始启动")
    # logger.debug('this is a debug message')
    # logger.warning('this is another debug message')
    # logger.error('this is another debug message')
    # logger.info('this is another debug message')
    # logger.success('this is success message!')
    # logger.critical('this is critical message!')
    # logger.add(sys.stdout, format="{message}")
    show_title("title", funcxxx=set_running)



    logger.critical('program is endding!')
    # logger.critical('=' * 100)
if __name__ == '__main__':
    main()
